Setup

Packages

suppressPackageStartupMessages({
  library(SingleCellExperiment)
  library(ggplot2)
  library(patchwork)
  library(caret)
  library(UpSetR)
})

theme_set(theme_bw())

Paths

module_base <- rprojroot::find_root(rprojroot::is_renv_project)
data_dir <- file.path(module_base, "scratch", "benchmark-datasets")
result_dir <- file.path(module_base, "results", "benchmark-results")

Functions

plot_pca_calls <- function(df, 
                           color_column, 
                           pred_column,
                           color_lab) {
  # Plot PCs colored by singlet/doublet, showing doublets on top
  # df is expected to contain columns PC1, PC2, `color_column`, and `pred_column`. These should _not_ be provided as strings
  ggplot(df) + 
    aes(x = PC1, 
        y = PC2, 
        color = {{color_column}}) +
  geom_point(
    size = 0.75, 
    alpha = 0.6
  ) +
  scale_color_manual(name = color_lab, values = c("black", "lightblue")) + 
  geom_point(
    data = dplyr::filter(df, {{color_column}} == "doublet"), 
    color = "black",
    size = 0.75
  ) +
  theme(
    legend.title.position = "top",
    legend.position = "bottom"
  )
}

plot_pca_metrics <- function(df, color_column, false_colors) {
  # Plot PCs colored by performance metric, showing false calls on top
  # false_colors is a vector of colors used for fn and fp points
  # df is expected to contain columns PC1, PC2, and `color_column`. This should _not_ be provided as a string.
  ggplot(df) + 
    aes(x = PC1, 
        y = PC2, 
        color = {{color_column}}) +
  geom_point(
    size = 0.75, 
    alpha = 0.6
  ) + 
  geom_point(
    data = dplyr::filter(df, {{color_column}} %in% false_colors), 
    size = 0.75
  ) +
  scale_color_identity() +
  theme(legend.position = "none")
}

Read and prepare input data

First, we’ll read in and combine doublet results into a list of data frames for each dataset. We’ll also two columns for each dataset:

  • consensus_call, which will be “doublet” if all methods predict “doublet,” and “singlet” otherwise
  • call_type, which will classify the consensus call as one of “tp”, “tn”, “fp”, or “fn” (true/false positive/negative)
# find all dataset names to process:
dataset_names <- list.files(result_dir, pattern = "*_scrublet.tsv") |>
  stringr::str_remove("_scrublet.tsv")
# used in PCA plots
confusion_colors <- c(
  "tp" = "lightblue",
  "tn" = "pink",
  "fp" = "blue",
  "fn" = "firebrick2"
)

# Read in and data for analysis
doublet_df_list <- dataset_names |>
  purrr::map(
    \(dataset) {
      
      scdbl_tsv <- file.path(result_dir, glue::glue("{dataset}_scdblfinder.tsv"))
      scrub_tsv <- file.path(result_dir, glue::glue("{dataset}_scrublet.tsv"))
      sce_file <- file.path(data_dir, dataset, glue::glue("{dataset}_sce.rds"))
      
      scdbl_df <- scdbl_tsv |>
        readr::read_tsv(show_col_types = FALSE) |>
        dplyr::select(
          barcodes,
          cxds_score, 
          scdbl_score = score, 
          scdbl_prediction  = class
        ) |>
        # add cxds calls at 0.75 threshold
        dplyr::mutate(
          cxds_prediction = dplyr::if_else(
            cxds_score >= 0.75,
            "doublet",
            "singlet"
          )
        ) 
      
      scrub_df <- readr::read_tsv(scrub_tsv, show_col_types = FALSE) 

      # grab ground truth and PCA coordinates
      sce <- readr::read_rds(sce_file)
      sce_df <- scuttle::makePerCellDF(sce, use.dimred = "PCA") |>
        tibble::rownames_to_column(var = "barcodes") |>
        dplyr::select(barcodes,
                      ground_truth = ground_truth_doublets, 
                      PC1 = PCA.1, 
                      PC2 = PCA.2)
      rm(sce)
      
      dataset_df <- scdbl_df |>
        dplyr::left_join(
          scrub_df, 
          by = "barcodes"
        ) |>
        dplyr::left_join(
          sce_df, 
          by = "barcodes"
        ) 
      
      # Add a consensus call column
      dataset_df <- dataset_df |>
        dplyr::rowwise() |>
        dplyr::mutate(consensus_call = dplyr::if_else(
          all(
            c(scdbl_prediction, scrublet_prediction, cxds_prediction) == "doublet"
          ),
          "doublet", 
          "singlet"
        )) |>
        dplyr::mutate(
          call_type = dplyr::case_when(
            consensus_call == "doublet" && ground_truth == "doublet" ~ "tp",
            consensus_call == "singlet" && ground_truth == "singlet" ~ "tn",
            consensus_call == "doublet" && ground_truth == "singlet" ~ "fp",
            consensus_call == "singlet" && ground_truth == "doublet" ~ "fn"
          ), 
          # set associated plotting colors
          call_type_color = dplyr::case_when(
            call_type == "tp" ~ unname(confusion_colors["tp"]),
            call_type == "tn" ~ unname(confusion_colors["tn"]),
            call_type == "fp" ~ unname(confusion_colors["fp"]),
            call_type == "fn" ~ unname(confusion_colors["fn"])
          )
          )
      
      return(dataset_df)
    }
  )
names(doublet_df_list) <- dataset_names

Performance metrics

This section shows performance metrics for the consensus calls for each dataset.

doublet_df_list |>
  purrr::iwalk( 
    \(df, dataset) {
        print(glue::glue("======================== {dataset} ========================"))
      
        cat("Table of consensus calls:")
        print(table(df$consensus_call))
        
        cat("\n\n")
        
        caret::confusionMatrix(
          # truth should be first
          table(
            "Truth" = df$ground_truth,
            "Consensus prediction" = df$consensus_call
          ), 
          positive = "doublet"
        ) |>
        print()
    }
  )
======================== hm-6k ========================
Table of consensus calls:
doublet singlet 
     62    6744 


Confusion Matrix and Statistics

         Consensus prediction
Truth     doublet singlet
  doublet      62     109
  singlet       0    6635
                                          
               Accuracy : 0.984           
                 95% CI : (0.9807, 0.9868)
    No Information Rate : 0.9909          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.5258          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 1.00000         
            Specificity : 0.98384         
         Pos Pred Value : 0.36257         
         Neg Pred Value : 1.00000         
             Prevalence : 0.00911         
         Detection Rate : 0.00911         
   Detection Prevalence : 0.02512         
      Balanced Accuracy : 0.99192         
                                          
       'Positive' Class : doublet         
                                          
======================== HMEC-orig-MULTI ========================
Table of consensus calls:
doublet singlet 
    247   26179 


Confusion Matrix and Statistics

         Consensus prediction
Truth     doublet singlet
  doublet     204    3364
  singlet      43   22815
                                         
               Accuracy : 0.8711         
                 95% CI : (0.867, 0.8751)
    No Information Rate : 0.9907         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.0911         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.825911       
            Specificity : 0.871500       
         Pos Pred Value : 0.057175       
         Neg Pred Value : 0.998119       
             Prevalence : 0.009347       
         Detection Rate : 0.007720       
   Detection Prevalence : 0.135019       
      Balanced Accuracy : 0.848705       
                                         
       'Positive' Class : doublet        
                                         
======================== pbmc-1B-dm ========================
Table of consensus calls:
doublet singlet 
     13    3777 


Confusion Matrix and Statistics

         Consensus prediction
Truth     doublet singlet
  doublet       8     122
  singlet       5    3655
                                         
               Accuracy : 0.9665         
                 95% CI : (0.9603, 0.972)
    No Information Rate : 0.9966         
    P-Value [Acc > NIR] : 1              
                                         
                  Kappa : 0.1063         
                                         
 Mcnemar's Test P-Value : <2e-16         
                                         
            Sensitivity : 0.615385       
            Specificity : 0.967699       
         Pos Pred Value : 0.061538       
         Neg Pred Value : 0.998634       
             Prevalence : 0.003430       
         Detection Rate : 0.002111       
   Detection Prevalence : 0.034301       
      Balanced Accuracy : 0.791542       
                                         
       'Positive' Class : doublet        
                                         
======================== pdx-MULTI ========================
Table of consensus calls:
doublet singlet 
      4   10292 


Confusion Matrix and Statistics

         Consensus prediction
Truth     doublet singlet
  doublet       3    1314
  singlet       1    8978
                                          
               Accuracy : 0.8723          
                 95% CI : (0.8657, 0.8787)
    No Information Rate : 0.9996          
    P-Value [Acc > NIR] : 1               
                                          
                  Kappa : 0.0038          
                                          
 Mcnemar's Test P-Value : <2e-16          
                                          
            Sensitivity : 0.7500000       
            Specificity : 0.8723280       
         Pos Pred Value : 0.0022779       
         Neg Pred Value : 0.9998886       
             Prevalence : 0.0003885       
         Detection Rate : 0.0002914       
   Detection Prevalence : 0.1279138       
      Balanced Accuracy : 0.8111640       
                                          
       'Positive' Class : doublet         
                                          

Visualizations

PCA

This section plots the PCA for each dataset, with three color schemes from left to right: - Ground truth doublets are shown in black - Consensus doublets are shown in black - Points are colored as tp, tn, fp, or fn based on comparing the consensus call to the ground truth

# Make a legend for the confusion-colored PCA
legend_plot <- data.frame(
  x = factor(names(confusion_colors), levels = names(confusion_colors)), y = 1:4
) |>
 ggplot(aes(x = x, y = y, color = x)) + 
  geom_point(size = 3) + 
  scale_color_manual(name = "Metric", values = confusion_colors) 
confusion_legend <- ggpubr::get_legend(legend_plot) |> ggpubr::as_ggplot()

doublet_df_list |>
  purrr::iwalk(
    \(df, dataset) {
      
      # First, ground truth
      p1 <- plot_pca_calls(
        df, 
        color_column = ground_truth, 
        color_lab = "Ground truth"
      )
      
      # Second, consensus call
      p2 <- plot_pca_calls(
        df, 
        color_column = consensus_call, 
        color_lab = "Consensus call"
      )
      
      # Third, call type
      p3 <- plot_pca_metrics(
        df,
        call_type_color,
        false_colors = unname(c(confusion_colors["fn"], confusion_colors["fp"]))
      )

      # combine and plot
      plot( p1 + p2 + p3 + confusion_legend + plot_annotation(glue::glue("PCA for {dataset}")) + plot_layout(ncol=4, widths = c(1,1,1,0.25)) )
    }
  )

Upset plots

This section shows upset plots for overlap among methods for each dataset.

pull_barcodes <- function(df, pred_var) {
  # Helper function to pull out barcodes for doublet calls
  df$barcodes[df[[pred_var]] == "doublet"]
}

upset_list <- doublet_df_list |>
  purrr::iwalk(
    \(df, dataset) {
      
      doublet_barcodes <- list(
        "scDblFinder" = pull_barcodes(df, "scdbl_prediction"),
        "scrublet"    = pull_barcodes(df, "scrublet_prediction"),
        "cxds"        = pull_barcodes(df, "cxds_prediction")
      )
      
      UpSetR::upset(fromList(doublet_barcodes), order.by = "freq") |> print()
      grid::grid.text(dataset,x = 0.65, y=0.95, gp=grid::gpar(fontsize=16)) # plot title

    }
  )

Session Info

# record the versions of the packages used in this analysis and other environment information
sessionInfo()
R version 4.4.0 (2024-04-24)
Platform: aarch64-apple-darwin20
Running under: macOS Sonoma 14.5

Matrix products: default
BLAS:   /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib 
LAPACK: /Library/Frameworks/R.framework/Versions/4.4-arm64/Resources/lib/libRlapack.dylib;  LAPACK version 3.12.0

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

time zone: America/New_York
tzcode source: internal

attached base packages:
[1] stats4    stats     graphics  grDevices datasets  utils     methods   base     

other attached packages:
 [1] UpSetR_1.4.0                caret_6.0-94                lattice_0.22-6              patchwork_1.2.0            
 [5] ggplot2_3.5.1               SingleCellExperiment_1.26.0 SummarizedExperiment_1.34.0 Biobase_2.64.0             
 [9] GenomicRanges_1.56.0        GenomeInfoDb_1.40.0         IRanges_2.38.0              S4Vectors_0.42.0           
[13] BiocGenerics_0.50.0         MatrixGenerics_1.16.0       matrixStats_1.3.0          

loaded via a namespace (and not attached):
  [1] pROC_1.18.5               gridExtra_2.3             rlang_1.1.3               magrittr_2.0.3            e1071_1.7-14             
  [6] compiler_4.4.0            DelayedMatrixStats_1.26.0 vctrs_0.6.5               reshape2_1.4.4            stringr_1.5.1            
 [11] pkgconfig_2.0.3           crayon_1.5.2              backports_1.4.1           XVector_0.44.0            labeling_0.4.3           
 [16] scuttle_1.14.0            utf8_1.2.4                prodlim_2023.08.28        tzdb_0.4.0                UCSC.utils_1.0.0         
 [21] bit_4.0.5                 purrr_1.0.2               xfun_0.43                 beachmat_2.20.0           zlibbioc_1.50.0          
 [26] jsonlite_1.8.8            recipes_1.0.10            DelayedArray_0.30.1       BiocParallel_1.38.0       broom_1.0.5              
 [31] parallel_4.4.0            R6_2.5.1                  stringi_1.8.4             car_3.1-2                 parallelly_1.37.1        
 [36] rpart_4.1.23              lubridate_1.9.3           Rcpp_1.0.12               iterators_1.0.14          knitr_1.46               
 [41] future.apply_1.11.2       readr_2.1.5               Matrix_1.7-0              splines_4.4.0             nnet_7.3-19              
 [46] timechange_0.3.0          tidyselect_1.2.1          rstudioapi_0.16.0         abind_1.4-5               yaml_2.3.8               
 [51] timeDate_4032.109         codetools_0.2-20          listenv_0.9.1             tibble_3.2.1              plyr_1.8.9               
 [56] withr_3.0.0               future_1.33.2             survival_3.5-8            proxy_0.4-27              ggpubr_0.6.0             
 [61] pillar_1.9.0              BiocManager_1.30.23       carData_3.0-5             renv_1.0.7                foreach_1.5.2            
 [66] generics_0.1.3            vroom_1.6.5               rprojroot_2.0.4           hms_1.1.3                 sparseMatrixStats_1.16.0 
 [71] munsell_0.5.1             scales_1.3.0              globals_0.16.3            class_7.3-22              glue_1.7.0               
 [76] tools_4.4.0               data.table_1.15.4         ggsignif_0.6.4            ModelMetrics_1.2.2.2      gower_1.0.1              
 [81] cowplot_1.1.3             grid_4.4.0                tidyr_1.3.1               ipred_0.9-14              colorspace_2.1-0         
 [86] nlme_3.1-164              GenomeInfoDbData_1.2.12   cli_3.6.2                 fansi_1.0.6               S4Arrays_1.4.0           
 [91] lava_1.8.0                dplyr_1.1.4               gtable_0.3.5              rstatix_0.7.2             digest_0.6.35            
 [96] SparseArray_1.4.3         farver_2.1.1              lifecycle_1.0.4           hardhat_1.3.1             httr_1.4.7               
[101] bit64_4.0.5               MASS_7.3-60.2            
LS0tCnRpdGxlOiAiQ29tcGFyaXNvbiBhbW9uZyBkb3VibGV0IHByZWRpY3Rpb25zIG9uIGJlbmNobWFya2luZyBkYXRhc2V0cyIKYXV0aG9yOiBTdGVwaGFuaWUgSi4gU3BpZWxtYW4KZGF0ZTogImByIFN5cy5EYXRlKClgIgpvdXRwdXQ6IAogIGh0bWxfbm90ZWJvb2s6CiAgICB0b2M6IHRydWUKICAgIHRvY19kZXB0aDogNAogICAgY29kZV9mb2xkaW5nOiBoaWRlCi0tLQoKIyMgU2V0dXAKCiMjIyBQYWNrYWdlcwoKCmBgYHtyIHBhY2thZ2VzfQpzdXBwcmVzc1BhY2thZ2VTdGFydHVwTWVzc2FnZXMoewogIGxpYnJhcnkoU2luZ2xlQ2VsbEV4cGVyaW1lbnQpCiAgbGlicmFyeShnZ3Bsb3QyKQogIGxpYnJhcnkocGF0Y2h3b3JrKQogIGxpYnJhcnkoY2FyZXQpCiAgbGlicmFyeShVcFNldFIpCn0pCgp0aGVtZV9zZXQodGhlbWVfYncoKSkKYGBgCgojIyMgUGF0aHMKCgpgYGB7ciBiYXNlIHBhdGhzfQptb2R1bGVfYmFzZSA8LSBycHJvanJvb3Q6OmZpbmRfcm9vdChycHJvanJvb3Q6OmlzX3JlbnZfcHJvamVjdCkKZGF0YV9kaXIgPC0gZmlsZS5wYXRoKG1vZHVsZV9iYXNlLCAic2NyYXRjaCIsICJiZW5jaG1hcmstZGF0YXNldHMiKQpyZXN1bHRfZGlyIDwtIGZpbGUucGF0aChtb2R1bGVfYmFzZSwgInJlc3VsdHMiLCAiYmVuY2htYXJrLXJlc3VsdHMiKQpgYGAKCiMjIyBGdW5jdGlvbnMKCmBgYHtyfQpwbG90X3BjYV9jYWxscyA8LSBmdW5jdGlvbihkZiwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbG9yX2NvbHVtbiwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHByZWRfY29sdW1uLAogICAgICAgICAgICAgICAgICAgICAgICAgICBjb2xvcl9sYWIpIHsKICAjIFBsb3QgUENzIGNvbG9yZWQgYnkgc2luZ2xldC9kb3VibGV0LCBzaG93aW5nIGRvdWJsZXRzIG9uIHRvcAogICMgZGYgaXMgZXhwZWN0ZWQgdG8gY29udGFpbiBjb2x1bW5zIFBDMSwgUEMyLCBgY29sb3JfY29sdW1uYCwgYW5kIGBwcmVkX2NvbHVtbmAuIFRoZXNlIHNob3VsZCBfbm90XyBiZSBwcm92aWRlZCBhcyBzdHJpbmdzCiAgZ2dwbG90KGRmKSArIAogICAgYWVzKHggPSBQQzEsIAogICAgICAgIHkgPSBQQzIsIAogICAgICAgIGNvbG9yID0ge3tjb2xvcl9jb2x1bW59fSkgKwogIGdlb21fcG9pbnQoCiAgICBzaXplID0gMC43NSwgCiAgICBhbHBoYSA9IDAuNgogICkgKwogIHNjYWxlX2NvbG9yX21hbnVhbChuYW1lID0gY29sb3JfbGFiLCB2YWx1ZXMgPSBjKCJibGFjayIsICJsaWdodGJsdWUiKSkgKyAKICBnZW9tX3BvaW50KAogICAgZGF0YSA9IGRwbHlyOjpmaWx0ZXIoZGYsIHt7Y29sb3JfY29sdW1ufX0gPT0gImRvdWJsZXQiKSwgCiAgICBjb2xvciA9ICJibGFjayIsCiAgICBzaXplID0gMC43NQogICkgKwogIHRoZW1lKAogICAgbGVnZW5kLnRpdGxlLnBvc2l0aW9uID0gInRvcCIsCiAgICBsZWdlbmQucG9zaXRpb24gPSAiYm90dG9tIgogICkKfQoKcGxvdF9wY2FfbWV0cmljcyA8LSBmdW5jdGlvbihkZiwgY29sb3JfY29sdW1uLCBmYWxzZV9jb2xvcnMpIHsKICAjIFBsb3QgUENzIGNvbG9yZWQgYnkgcGVyZm9ybWFuY2UgbWV0cmljLCBzaG93aW5nIGZhbHNlIGNhbGxzIG9uIHRvcAogICMgZmFsc2VfY29sb3JzIGlzIGEgdmVjdG9yIG9mIGNvbG9ycyB1c2VkIGZvciBmbiBhbmQgZnAgcG9pbnRzCiAgIyBkZiBpcyBleHBlY3RlZCB0byBjb250YWluIGNvbHVtbnMgUEMxLCBQQzIsIGFuZCBgY29sb3JfY29sdW1uYC4gVGhpcyBzaG91bGQgX25vdF8gYmUgcHJvdmlkZWQgYXMgYSBzdHJpbmcuCiAgZ2dwbG90KGRmKSArIAogICAgYWVzKHggPSBQQzEsIAogICAgICAgIHkgPSBQQzIsIAogICAgICAgIGNvbG9yID0ge3tjb2xvcl9jb2x1bW59fSkgKwogIGdlb21fcG9pbnQoCiAgICBzaXplID0gMC43NSwgCiAgICBhbHBoYSA9IDAuNgogICkgKyAKICBnZW9tX3BvaW50KAogICAgZGF0YSA9IGRwbHlyOjpmaWx0ZXIoZGYsIHt7Y29sb3JfY29sdW1ufX0gJWluJSBmYWxzZV9jb2xvcnMpLCAKICAgIHNpemUgPSAwLjc1CiAgKSArCiAgc2NhbGVfY29sb3JfaWRlbnRpdHkoKSArCiAgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiKQp9CmBgYAoKCiMjIyBSZWFkIGFuZCBwcmVwYXJlIGlucHV0IGRhdGEKCkZpcnN0LCB3ZSdsbCByZWFkIGluIGFuZCBjb21iaW5lIGRvdWJsZXQgcmVzdWx0cyBpbnRvIGEgbGlzdCBvZiBkYXRhIGZyYW1lcyBmb3IgZWFjaCBkYXRhc2V0LgpXZSdsbCBhbHNvIHR3byBjb2x1bW5zIGZvciBlYWNoIGRhdGFzZXQ6CgotIGBjb25zZW5zdXNfY2FsbGAsIHdoaWNoIHdpbGwgYmUgImRvdWJsZXQiIGlmIF9hbGxfIG1ldGhvZHMgcHJlZGljdCAiZG91YmxldCwiIGFuZCAic2luZ2xldCIgb3RoZXJ3aXNlCi0gYGNhbGxfdHlwZWAsIHdoaWNoIHdpbGwgY2xhc3NpZnkgdGhlIGNvbnNlbnN1cyBjYWxsIGFzIG9uZSBvZiAidHAiLCAidG4iLCAiZnAiLCBvciAiZm4iICh0cnVlL2ZhbHNlIHBvc2l0aXZlL25lZ2F0aXZlKSAKCmBgYHtyIHBhdGhzfQojIGZpbmQgYWxsIGRhdGFzZXQgbmFtZXMgdG8gcHJvY2VzczoKZGF0YXNldF9uYW1lcyA8LSBsaXN0LmZpbGVzKHJlc3VsdF9kaXIsIHBhdHRlcm4gPSAiKl9zY3J1YmxldC50c3YiKSB8PgogIHN0cmluZ3I6OnN0cl9yZW1vdmUoIl9zY3J1YmxldC50c3YiKQpgYGAKCmBgYHtyIHJlYWRfZGF0YX0KIyB1c2VkIGluIFBDQSBwbG90cwpjb25mdXNpb25fY29sb3JzIDwtIGMoCiAgInRwIiA9ICJsaWdodGJsdWUiLAogICJ0biIgPSAicGluayIsCiAgImZwIiA9ICJibHVlIiwKICAiZm4iID0gImZpcmVicmljazIiCikKCiMgUmVhZCBpbiBhbmQgZGF0YSBmb3IgYW5hbHlzaXMKZG91YmxldF9kZl9saXN0IDwtIGRhdGFzZXRfbmFtZXMgfD4KICBwdXJycjo6bWFwKAogICAgXChkYXRhc2V0KSB7CiAgICAgIAogICAgICBzY2RibF90c3YgPC0gZmlsZS5wYXRoKHJlc3VsdF9kaXIsIGdsdWU6OmdsdWUoIntkYXRhc2V0fV9zY2RibGZpbmRlci50c3YiKSkKICAgICAgc2NydWJfdHN2IDwtIGZpbGUucGF0aChyZXN1bHRfZGlyLCBnbHVlOjpnbHVlKCJ7ZGF0YXNldH1fc2NydWJsZXQudHN2IikpCiAgICAgIHNjZV9maWxlIDwtIGZpbGUucGF0aChkYXRhX2RpciwgZGF0YXNldCwgZ2x1ZTo6Z2x1ZSgie2RhdGFzZXR9X3NjZS5yZHMiKSkKICAgICAgCiAgICAgIHNjZGJsX2RmIDwtIHNjZGJsX3RzdiB8PgogICAgICAgIHJlYWRyOjpyZWFkX3RzdihzaG93X2NvbF90eXBlcyA9IEZBTFNFKSB8PgogICAgICAgIGRwbHlyOjpzZWxlY3QoCiAgICAgICAgICBiYXJjb2RlcywKICAgICAgICAgIGN4ZHNfc2NvcmUsIAogICAgICAgICAgc2NkYmxfc2NvcmUgPSBzY29yZSwgCiAgICAgICAgICBzY2RibF9wcmVkaWN0aW9uICA9IGNsYXNzCiAgICAgICAgKSB8PgogICAgICAgICMgYWRkIGN4ZHMgY2FsbHMgYXQgMC43NSB0aHJlc2hvbGQKICAgICAgICBkcGx5cjo6bXV0YXRlKAogICAgICAgICAgY3hkc19wcmVkaWN0aW9uID0gZHBseXI6OmlmX2Vsc2UoCiAgICAgICAgICAgIGN4ZHNfc2NvcmUgPj0gMC43NSwKICAgICAgICAgICAgImRvdWJsZXQiLAogICAgICAgICAgICAic2luZ2xldCIKICAgICAgICAgICkKICAgICAgICApIAogICAgICAKICAgICAgc2NydWJfZGYgPC0gcmVhZHI6OnJlYWRfdHN2KHNjcnViX3Rzdiwgc2hvd19jb2xfdHlwZXMgPSBGQUxTRSkgCgogICAgICAjIGdyYWIgZ3JvdW5kIHRydXRoIGFuZCBQQ0EgY29vcmRpbmF0ZXMKICAgICAgc2NlIDwtIHJlYWRyOjpyZWFkX3JkcyhzY2VfZmlsZSkKICAgICAgc2NlX2RmIDwtIHNjdXR0bGU6Om1ha2VQZXJDZWxsREYoc2NlLCB1c2UuZGltcmVkID0gIlBDQSIpIHw+CiAgICAgICAgdGliYmxlOjpyb3duYW1lc190b19jb2x1bW4odmFyID0gImJhcmNvZGVzIikgfD4KICAgICAgICBkcGx5cjo6c2VsZWN0KGJhcmNvZGVzLAogICAgICAgICAgICAgICAgICAgICAgZ3JvdW5kX3RydXRoID0gZ3JvdW5kX3RydXRoX2RvdWJsZXRzLCAKICAgICAgICAgICAgICAgICAgICAgIFBDMSA9IFBDQS4xLCAKICAgICAgICAgICAgICAgICAgICAgIFBDMiA9IFBDQS4yKQogICAgICBybShzY2UpCiAgICAgIAogICAgICBkYXRhc2V0X2RmIDwtIHNjZGJsX2RmIHw+CiAgICAgICAgZHBseXI6OmxlZnRfam9pbigKICAgICAgICAgIHNjcnViX2RmLCAKICAgICAgICAgIGJ5ID0gImJhcmNvZGVzIgogICAgICAgICkgfD4KICAgICAgICBkcGx5cjo6bGVmdF9qb2luKAogICAgICAgICAgc2NlX2RmLCAKICAgICAgICAgIGJ5ID0gImJhcmNvZGVzIgogICAgICAgICkgCiAgICAgIAogICAgICAjIEFkZCBhIGNvbnNlbnN1cyBjYWxsIGNvbHVtbgogICAgICBkYXRhc2V0X2RmIDwtIGRhdGFzZXRfZGYgfD4KICAgICAgICBkcGx5cjo6cm93d2lzZSgpIHw+CiAgICAgICAgZHBseXI6Om11dGF0ZShjb25zZW5zdXNfY2FsbCA9IGRwbHlyOjppZl9lbHNlKAogICAgICAgICAgYWxsKAogICAgICAgICAgICBjKHNjZGJsX3ByZWRpY3Rpb24sIHNjcnVibGV0X3ByZWRpY3Rpb24sIGN4ZHNfcHJlZGljdGlvbikgPT0gImRvdWJsZXQiCiAgICAgICAgICApLAogICAgICAgICAgImRvdWJsZXQiLCAKICAgICAgICAgICJzaW5nbGV0IgogICAgICAgICkpIHw+CiAgICAgICAgZHBseXI6Om11dGF0ZSgKICAgICAgICAgIGNhbGxfdHlwZSA9IGRwbHlyOjpjYXNlX3doZW4oCiAgICAgICAgICAgIGNvbnNlbnN1c19jYWxsID09ICJkb3VibGV0IiAmJiBncm91bmRfdHJ1dGggPT0gImRvdWJsZXQiIH4gInRwIiwKICAgICAgICAgICAgY29uc2Vuc3VzX2NhbGwgPT0gInNpbmdsZXQiICYmIGdyb3VuZF90cnV0aCA9PSAic2luZ2xldCIgfiAidG4iLAogICAgICAgICAgICBjb25zZW5zdXNfY2FsbCA9PSAiZG91YmxldCIgJiYgZ3JvdW5kX3RydXRoID09ICJzaW5nbGV0IiB+ICJmcCIsCiAgICAgICAgICAgIGNvbnNlbnN1c19jYWxsID09ICJzaW5nbGV0IiAmJiBncm91bmRfdHJ1dGggPT0gImRvdWJsZXQiIH4gImZuIgogICAgICAgICAgKSwgCiAgICAgICAgICAjIHNldCBhc3NvY2lhdGVkIHBsb3R0aW5nIGNvbG9ycwogICAgICAgICAgY2FsbF90eXBlX2NvbG9yID0gZHBseXI6OmNhc2Vfd2hlbigKICAgICAgICAgICAgY2FsbF90eXBlID09ICJ0cCIgfiB1bm5hbWUoY29uZnVzaW9uX2NvbG9yc1sidHAiXSksCiAgICAgICAgICAgIGNhbGxfdHlwZSA9PSAidG4iIH4gdW5uYW1lKGNvbmZ1c2lvbl9jb2xvcnNbInRuIl0pLAogICAgICAgICAgICBjYWxsX3R5cGUgPT0gImZwIiB+IHVubmFtZShjb25mdXNpb25fY29sb3JzWyJmcCJdKSwKICAgICAgICAgICAgY2FsbF90eXBlID09ICJmbiIgfiB1bm5hbWUoY29uZnVzaW9uX2NvbG9yc1siZm4iXSkKICAgICAgICAgICkKICAgICAgICAgICkKICAgICAgCiAgICAgIHJldHVybihkYXRhc2V0X2RmKQogICAgfQogICkKbmFtZXMoZG91YmxldF9kZl9saXN0KSA8LSBkYXRhc2V0X25hbWVzCmBgYAoKCiMjIFBlcmZvcm1hbmNlIG1ldHJpY3MKClRoaXMgc2VjdGlvbiBzaG93cyBwZXJmb3JtYW5jZSBtZXRyaWNzIGZvciB0aGUgY29uc2Vuc3VzIGNhbGxzIGZvciBlYWNoIGRhdGFzZXQuCgpgYGB7cn0KZG91YmxldF9kZl9saXN0IHw+CiAgcHVycnI6Oml3YWxrKCAKICAgIFwoZGYsIGRhdGFzZXQpIHsKICAgICAgICBwcmludChnbHVlOjpnbHVlKCI9PT09PT09PT09PT09PT09PT09PT09PT0ge2RhdGFzZXR9ID09PT09PT09PT09PT09PT09PT09PT09PSIpKQogICAgICAKICAgICAgICBjYXQoIlRhYmxlIG9mIGNvbnNlbnN1cyBjYWxsczoiKQogICAgICAgIHByaW50KHRhYmxlKGRmJGNvbnNlbnN1c19jYWxsKSkKICAgICAgICAKICAgICAgICBjYXQoIlxuXG4iKQogICAgICAgIAogICAgICAgIGNhcmV0Ojpjb25mdXNpb25NYXRyaXgoCiAgICAgICAgICAjIHRydXRoIHNob3VsZCBiZSBmaXJzdAogICAgICAgICAgdGFibGUoCiAgICAgICAgICAgICJUcnV0aCIgPSBkZiRncm91bmRfdHJ1dGgsCiAgICAgICAgICAgICJDb25zZW5zdXMgcHJlZGljdGlvbiIgPSBkZiRjb25zZW5zdXNfY2FsbAogICAgICAgICAgKSwgCiAgICAgICAgICBwb3NpdGl2ZSA9ICJkb3VibGV0IgogICAgICAgICkgfD4KICAgICAgICBwcmludCgpCiAgICB9CiAgKQpgYGAKCgoKIyMgVmlzdWFsaXphdGlvbnMKCiMjIyBQQ0EKClRoaXMgc2VjdGlvbiBwbG90cyB0aGUgUENBIGZvciBlYWNoIGRhdGFzZXQsIHdpdGggdGhyZWUgY29sb3Igc2NoZW1lcyBmcm9tIGxlZnQgdG8gcmlnaHQ6Ci0gR3JvdW5kIHRydXRoIGRvdWJsZXRzIGFyZSBzaG93biBpbiBibGFjawotIENvbnNlbnN1cyBkb3VibGV0cyBhcmUgc2hvd24gaW4gYmxhY2sKLSBQb2ludHMgYXJlIGNvbG9yZWQgYXMgdHAsIHRuLCBmcCwgb3IgZm4gYmFzZWQgb24gY29tcGFyaW5nIHRoZSBjb25zZW5zdXMgY2FsbCB0byB0aGUgZ3JvdW5kIHRydXRoCgoKYGBge3IsIGZpZy53aWR0aCA9IDEyLCBmaWcuaGVpZ2h0ID0gNn0KIyBNYWtlIGEgbGVnZW5kIGZvciB0aGUgY29uZnVzaW9uLWNvbG9yZWQgUENBCmxlZ2VuZF9wbG90IDwtIGRhdGEuZnJhbWUoCiAgeCA9IGZhY3RvcihuYW1lcyhjb25mdXNpb25fY29sb3JzKSwgbGV2ZWxzID0gbmFtZXMoY29uZnVzaW9uX2NvbG9ycykpLCB5ID0gMTo0CikgfD4KIGdncGxvdChhZXMoeCA9IHgsIHkgPSB5LCBjb2xvciA9IHgpKSArIAogIGdlb21fcG9pbnQoc2l6ZSA9IDMpICsgCiAgc2NhbGVfY29sb3JfbWFudWFsKG5hbWUgPSAiTWV0cmljIiwgdmFsdWVzID0gY29uZnVzaW9uX2NvbG9ycykgCmNvbmZ1c2lvbl9sZWdlbmQgPC0gZ2dwdWJyOjpnZXRfbGVnZW5kKGxlZ2VuZF9wbG90KSB8PiBnZ3B1YnI6OmFzX2dncGxvdCgpCgpkb3VibGV0X2RmX2xpc3QgfD4KICBwdXJycjo6aXdhbGsoCiAgICBcKGRmLCBkYXRhc2V0KSB7CiAgICAgIAogICAgICAjIEZpcnN0LCBncm91bmQgdHJ1dGgKICAgICAgcDEgPC0gcGxvdF9wY2FfY2FsbHMoCiAgICAgICAgZGYsIAogICAgICAgIGNvbG9yX2NvbHVtbiA9IGdyb3VuZF90cnV0aCwgCiAgICAgICAgY29sb3JfbGFiID0gIkdyb3VuZCB0cnV0aCIKICAgICAgKQogICAgICAKICAgICAgIyBTZWNvbmQsIGNvbnNlbnN1cyBjYWxsCiAgICAgIHAyIDwtIHBsb3RfcGNhX2NhbGxzKAogICAgICAgIGRmLCAKICAgICAgICBjb2xvcl9jb2x1bW4gPSBjb25zZW5zdXNfY2FsbCwgCiAgICAgICAgY29sb3JfbGFiID0gIkNvbnNlbnN1cyBjYWxsIgogICAgICApCiAgICAgIAogICAgICAjIFRoaXJkLCBjYWxsIHR5cGUKICAgICAgcDMgPC0gcGxvdF9wY2FfbWV0cmljcygKICAgICAgICBkZiwKICAgICAgICBjYWxsX3R5cGVfY29sb3IsCiAgICAgICAgZmFsc2VfY29sb3JzID0gdW5uYW1lKGMoY29uZnVzaW9uX2NvbG9yc1siZm4iXSwgY29uZnVzaW9uX2NvbG9yc1siZnAiXSkpCiAgICAgICkKCiAgICAgICMgY29tYmluZSBhbmQgcGxvdAogICAgICBwbG90KCBwMSArIHAyICsgcDMgKyBjb25mdXNpb25fbGVnZW5kICsgcGxvdF9hbm5vdGF0aW9uKGdsdWU6OmdsdWUoIlBDQSBmb3Ige2RhdGFzZXR9IikpICsgcGxvdF9sYXlvdXQobmNvbD00LCB3aWR0aHMgPSBjKDEsMSwxLDAuMjUpKSApCiAgICB9CiAgKQpgYGAKCgojIyMgVXBzZXQgcGxvdHMKClRoaXMgc2VjdGlvbiBzaG93cyB1cHNldCBwbG90cyBmb3Igb3ZlcmxhcCBhbW9uZyBtZXRob2RzIGZvciBlYWNoIGRhdGFzZXQuCgpgYGB7cn0KcHVsbF9iYXJjb2RlcyA8LSBmdW5jdGlvbihkZiwgcHJlZF92YXIpIHsKICAjIEhlbHBlciBmdW5jdGlvbiB0byBwdWxsIG91dCBiYXJjb2RlcyBmb3IgZG91YmxldCBjYWxscwogIGRmJGJhcmNvZGVzW2RmW1twcmVkX3Zhcl1dID09ICJkb3VibGV0Il0KfQoKdXBzZXRfbGlzdCA8LSBkb3VibGV0X2RmX2xpc3QgfD4KICBwdXJycjo6aXdhbGsoCiAgICBcKGRmLCBkYXRhc2V0KSB7CiAgICAgIAogICAgICBkb3VibGV0X2JhcmNvZGVzIDwtIGxpc3QoCiAgICAgICAgInNjRGJsRmluZGVyIiA9IHB1bGxfYmFyY29kZXMoZGYsICJzY2RibF9wcmVkaWN0aW9uIiksCiAgICAgICAgInNjcnVibGV0IiAgICA9IHB1bGxfYmFyY29kZXMoZGYsICJzY3J1YmxldF9wcmVkaWN0aW9uIiksCiAgICAgICAgImN4ZHMiICAgICAgICA9IHB1bGxfYmFyY29kZXMoZGYsICJjeGRzX3ByZWRpY3Rpb24iKQogICAgICApCiAgICAgIAogICAgICBVcFNldFI6OnVwc2V0KGZyb21MaXN0KGRvdWJsZXRfYmFyY29kZXMpLCBvcmRlci5ieSA9ICJmcmVxIikgfD4gcHJpbnQoKQogICAgICBncmlkOjpncmlkLnRleHQoICMgcGxvdCB0aXRsZQogICAgICAgIGRhdGFzZXQsCiAgICAgICAgeCA9IDAuNjUsIAogICAgICAgIHkgPSAwLjk1LCAKICAgICAgICBncCA9IGdyaWQ6OmdwYXIoZm9udHNpemU9MTYpCiAgICAgICkgCgogICAgfQogICkKCmBgYAoKCiMjIFNlc3Npb24gSW5mbwoKYGBge3Igc2Vzc2lvbiBpbmZvfQojIHJlY29yZCB0aGUgdmVyc2lvbnMgb2YgdGhlIHBhY2thZ2VzIHVzZWQgaW4gdGhpcyBhbmFseXNpcyBhbmQgb3RoZXIgZW52aXJvbm1lbnQgaW5mb3JtYXRpb24Kc2Vzc2lvbkluZm8oKQpgYGAK